package com.micro.util;/*
 * 文件名：Test2.java
 * 版权：Copyright 2006-2013 Huawei Tech. Co. Ltd. All Rights Reserved. 
 * 描述： Test2.java
 * 修改人：huang.song
 * 修改时间：Jul 17, 2013
 * 修改内容：新增
 */

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.List;

import com.micro.model.Subject;
import com.micro.service.SubjectService;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

/**
 * @author huang.song
 * @version TV VAS V100R001 Jul 17, 2013
 * @see [相关类/方法]
 * @since [产品/模块版本]
 */
public class Spider
{
    /**
     * 爬取页面总数
     */
    private static int SPIDERCOUNT = 0;

    private static final String TITLE = "bname_content";

    private static final String CONTENT = "content";

    private static final String NEXTPAGEID = "bottem2";

    private static final String SPIDERURL = "http://www.biqugeg.com";

    private static final String CHAPTERONEURL = "/23_23680/12463561.html";

    public static void main(String[] args)
    {
        spider(CHAPTERONEURL);
    }

    public static List<Subject> subjectsList = new ArrayList<Subject>();

    private static ApplicationContext context = new ClassPathXmlApplicationContext(new String[]{"classpath:conf/spring.xml"
            ,"classpath:conf/spring-mybatis.xml"});
    private static SubjectService subjectService = (SubjectService) context.getBean("subjectServiceImpl");
    /**
     * 爬取网页文字
     *
     * @param url 网页地址
     */
    public static void spider(String url)
    {
        Subject subject = new Subject();
        String storyUrl = SPIDERURL;
        String spiderUrl = storyUrl + url;
        String nextUrl = null;
        Document doc;
        Connection connect = null;
        try
        {
            Thread.sleep(200);

            connect = Jsoup.connect(spiderUrl);
            connect.timeout(Integer.MAX_VALUE);
            doc = connect.get();

            //<a class="s xst" onclick="atarget(this)" href="http://bbs.ztesoft.com/forum.php?mod=viewthread&tid=1314&extra=page%3D1">
            Element title = doc.getElementsByTag("h1").get(0);
            Element content = doc.getElementById(CONTENT);
            Elements elementA = doc.getElementsByClass(NEXTPAGEID);

            Elements select = elementA.select("a");

            String value = null;
            for (Element element : select)
            {
                value = element.childNodes().get(0).toString().trim();
                if (value.equals("← 上一章") || value.equals("下一章 →") || value.equals("下一章"))
                {
                    nextUrl = element.attr("href");
                }
            }

            String contentTxt = null;
            // 按照Id无法获取内容，则按照class获取
            if (null == content)
            {
                Elements contentEle = doc.getElementsByClass(CONTENT);
                contentTxt = contentEle.html();
            }
            else
            {
                // 按照内容的Id抓取内容
                contentTxt = content.html();
            }
            //contentTxt = contentTxt.replace("<br />", "\r\n");
            contentTxt = contentTxt.replace("&nbsp;", "");
            contentTxt = contentTxt.replace("<P>", "");
            contentTxt = contentTxt.replace("<p>", "");


            // 设置章节名称
            String chapterName = null;
            if (null == title)
            {
                Elements titleEle = doc.getElementsByClass(TITLE);
                chapterName = titleEle.html();
            }
            else
            {
                chapterName = title.html();
            }
            subject.setSubjectId(UUIDTool.getUUID());
            subject.setSubjectTitle(chapterName);
            subject.setSubjectLink(spiderUrl);
            subject.setSubjectContent(contentTxt);
            subject.setpSubjectId("-1");
            subject.setSubjectContentByte(subject.getSubjectContent().getBytes("UTF-8"));
//            subject.setSubjectCatName("交管产品长研知识库");
            subject.setUserName("匿名");
            subject.setSubjectTop("0");
            subject.setSubjectLaud(0);

            int isExist = subjectService.querySubjectIsExistByTitle(chapterName);
            if(isExist == 0) {
                subjectService.insertSubject(subject);
            } else {
                System.out.println("已存在相同主题数据，跳过........");
            }

            // 再次爬取下一个网页
            if (!(StringUtil.isBlank(nextUrl) || nextUrl.startsWith("http") || nextUrl.startsWith("www")))
            {
                SPIDERCOUNT++;
                System.out.println("爬取页面:" + title + "，正在爬取下一个网页：" + nextUrl);
                spider(nextUrl);
            }
        }
        catch (Exception e)
        {
            e.printStackTrace();
        }
    }
}
